'''
Created on 2011-10-26

@author: Administrator
'''
import os
import urllib2
import sqlite3
import datetime
import threading
import Queue
import time
import shutil 
from setting import *
from packStockData.stocklist import StockList,StockListDB
from packStockData.stockdata import StockDataDB,StockDataCalc,StockByDateDB,StockWeek

class ThreadStockDataDB(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)
        self.queue=urlqueue
    def run(self):
        job=self.queue.get()        
        sd=StockDataDB(job)
        sd.updatestockdata()
        self.queue.task_done()
class ThreadStockDataDownload(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)
        self.queue=urlqueue
    def run(self):
        job=self.queue.get()        
        stockcode=job
        if job.find('HK')>0:stockcode=job.strip('0')
#        print 'thread stockcode:',stockcode
        url='http://www.bloomberg.com/apps/data?pid=webpxta&Securities='+stockcode+'&TimePeriod=5Y&Outfields=HDATE,PR006-H,PR005-H,PR007-H,PR008-H'
        stockcode2=job.replace(':','_')
        try:
            content=urllib2.urlopen(url).read()
        except:
            print '[except]',#url   
            fname=os.path.join(errordatapath,stockcode2)
            f=open(fname,'w')
#            f.write(content)
            f.close()        
            self.queue.task_done()  
            return 
        if content[:5]=='Error':
            print '[Error]',#url
            fname=os.path.join(errordatapath,stockcode2)
            f=open(fname,'w')
#            f.write(content)
            f.close() 
            self.queue.task_done()
        else:
            fname=os.path.join(rawcontentpath,stockcode2)
            f=open(fname,'w')
            f.write(content)
            f.close()  
            filetobedelete=os.path.join(jobtodopath,stockcode2)
            if os.path.exists(filetobedelete):os.remove(filetobedelete)
            self.queue.task_done()
class ThreadMain(threading.Thread):
    def __init__(self,num):
        threading.Thread.__init__(self)
        self.num=num
    def run(self):          
#        print 'g_jobsnumber',self.num
        for i in range(self.num):
            dt=ThreadStockDataDownload()
            dt.start()
        urlqueue.join()

class stock2011():
    def __init__(self):
        if not os.path.exists(rawcontentpath):os.mkdir(rawcontentpath)
        if not os.path.exists(errordatapath):os.mkdir(errordatapath)
        if not os.path.exists(jobtodopath):os.mkdir(jobtodopath)
        if not os.path.exists(stockbycodepath):os.mkdir(stockbycodepath)
        if not os.path.exists(stockbydatepath):os.mkdir(stockbydatepath)
        for reg in reglist:
            dbpath=os.path.join(stockbycodepath,reg)
            if not os.path.exists(dbpath):os.mkdir(dbpath)
#            self.clearpathfile(dbpath)
        for reg in reglist:
            bydatepath=os.path.join(stockbydatepath,reg)
            if not os.path.exists(bydatepath):os.mkdir(bydatepath)
    def clearpathfile(self,mypath):
        for root,dirs,files in os.walk(mypath):
            for f in files:
                os.remove(os.path.join(root,f))  
    def geterrorjobslist(self):
        errorlist=self.getpathfilelist(errordatapath)
        return errorlist
    def getpathfilelist(self,mypath,filter=''):
        flist=[]
        for root,dirs,files in os.walk(mypath):
            for f in files:
                if filter=='':flist.append(f.replace('_',':'))
                else:
                    if f.find(filter)>0:flist.append(f.replace('_',':'))
        return flist     
    def dostockdb2(self):
        jobslist=self.getpathfilelist(rawcontentpath) 
        i=0
        for job in jobslist:
            i=i+1
            
            sd=StockDataDB(job.replace(':','_'))
            sd.updatestockdata()  
            print i,job       
    def dostockdb(self):
        jobslist=self.getpathfilelist(rawcontentpath)
        print 'todo job number:',len(jobslist)  
        while len(jobslist)>workers_number:            
            tlist=[]
            tlist=jobslist[:workers_number]
            jobslist=jobslist[workers_number:]
            print 'jobslist len:',len(jobslist),datetime.datetime.now()
            self.dostockdbthread(tlist)
        if len(jobslist)>0:self.dostockdbthread(jobslist)
            
    def dostockdbthread(self,todolist):
        jobsnum=len(todolist)
#        print 'jobsnum:',jobsnum
#        print 'this round:',jobsnum#,str(datetime.datetime.now()),self.todolist
        for job in todolist[:jobsnum]:
            urlqueue.put(job.replace(':',"_"),block=True ,timeout=threadtimeout)              
        for i in range(jobsnum):
            dt=ThreadStockDataDB()
            dt.start()
        urlqueue.join()
        return jobsnum               
    def downloadstart(self):
        self.clearpathfile(rawcontentpath)
        self.clearpathfile(errordatapath)
        self.clearpathfile(jobtodopath)
        jobs=StockListDB()
        jobslist=jobs.getjobslist()

        for job in jobslist.keys():
#            print job,jobslist[job] 
                   
            if jobslist[job]==str(datetime.date.today()):
                pass
            else:
                
                fname=os.path.join(jobtodopath,job.replace(':','_'))
                f=open(fname,'w')
                f.close()   

        todolist=[]
        todolist=self.getsomejobs()
        g_jobsnumber=len(todolist) 
#        print 'this round todo:',len(todolist)
        while g_jobsnumber>0:
            print 'this round todo:',g_jobsnumber,datetime.datetime.now()
            for job in todolist:
                urlqueue.put(job,block=True ,timeout= threadtimeout)             
            td=ThreadMain(g_jobsnumber)
            td.start()
            td.join(10)
            todolist=self.getsomejobs()
            g_jobsnumber=len(todolist) 

    def getsomejobs(self):
        todolist=self.getpathfilelist(jobtodopath)
        errorlist=self.geterrorjobslist()
        self.clearpathfile(errordatapath)
#        print todolist
#        print errorlist
#        print todolist==errorlist
        if todolist==errorlist:return []
        else:
            
            jobnum=len(todolist)
#            print 'go here',jobnum,workers_number,jobnum<=workers_number
            if jobnum<=workers_number:
                return todolist
            else:
#                print todolist[:workers_number]
                return todolist[:workers_number]

    def checkstockfile(self):
        rawlist=[]
        for root,dirs,files in os.walk(rawcontentpath):
            for f in files:
                rawlist.append(f)
                fname=os.path.join(root,f)
                f=open(fname,'r')
                content=f.read()
                f.close()
                tlist=content.split('\n')
                s=tlist[0].find('Count')
                p=tlist[0].find('=',s)
                q=tlist[0].find('"',p)
                num=int(tlist[0][p+1:q])
                flines=len(tlist)
                if flines==num+3:pass
                else:print 'checkstockfile failed:',len(tlist),num,fname
                
        print 'raw file num:',len(rawlist),rawlist[0]
        dblist=[]
        for reg in reglist:
            rlist=self.getpathfilelist(os.path.join(stockbycodepath,reg))
            for r in rlist:
                p=r.find('.')
                dblist.append(r[:p]+'_'+reg)
        print 'db file num:',len(dblist),dblist[0]
        
        for raw in rawlist:
            if raw in dblist:pass
            else:print raw,'not in dblist'
        return
        for reg in reglist:
            stocklist=self.getpathfilelist(os.path.join(stockbycodepath,reg))
            print reg,len(stocklist)
            for stock in stocklist:
#                print stock
                if stock.find('.')>0:stock=stock[:stock.find('.')]
                sd=StockDataDB(stock+'_'+reg)
                dlist=sd.loaddata('kday')  
#                print stock,'loaddata:',len(dlist)
                if dlist is None:print 'data is None:',stock   
#                else:print stock,' is ok'    
   
    def getdatabydate(self):
        stocklist=self.getpathfilelist(rawcontentpath)
        somedate='20111103'
        fname='c:\\'+somedate+'.txt'
        f=open(fname,'w')
        for stock in stocklist:
            sd=StockDataDB(stock)
            dlist=sd.getdatabydate(somedate)
            if dlist is None:pass
            else:
                print stock,len(dlist)
                tstr=''
                for d in dlist:
                    tstr=tstr+str(d)+'|'                    
                f.write( tstr+'\n')
        f.close()
    def trans2bydate(self):
        if not os.path.exists(stockbydatepath) :os.mkdir(stockbydatepath)
        
        for reg in reglist:
            datelist=[]
            for root,dirs,files in os.walk(os.path.join(stockbydatepath,reg)):
                for f in files:
                    p=f.find('.')
                    datelist.append(int(f[:p]))
            print reg,len(datelist)
            if len(datelist)==0:lastdate=defaultfirstdate
            else:lastdate=max(datelist)
            print 'lastdate:',lastdate
            #################################
            stockbydate={}
            stockbyweek={}
            for d in range(lastdate,20500000):
                stockbydate[d]=[]  
                stockbyweek[d]=[]
            stocklist=self.getpathfilelist(os.path.join(stockbycodepath,reg))
            for stock in stocklist:
                if stock.find('.')>0:stock=stock[:stock.find('.')]
                sd=StockDataDB(stock+'_'+reg)
                dlist=sd.loaddata('kday',lastdate)  
                wlist=sd.loaddata('kweek',lastdate)
#                print 'loaddata:',len(dlist)
                if dlist is None:pass
                else:
                    #################################
                    datalist={}
                    for r in dlist:
                        datalist[r[0]]=r
                        datalist[r[0]][0]=stock
                    for data in datalist.keys():
                        stockbydate[data].append(datalist[data])  
                    ########################################
                    wdatalist={}
                    for r in wlist:
                        wdatalist[r[0]]=r
                        wdatalist[r[0]][0]=stock
                    for data in wdatalist.keys():
                        stockbyweek[data].append(wdatalist[data]) 
                    ########################################
                                
            for sdate in stockbydate.keys():
                if len(stockbydate[sdate])>0:
                    print sdate
                    sdb=StockByDateDB(sdate,reg)
                    sdb.updatestockdata('stocklistday',stockbydate[sdate])
                    stockbydate[sdate]=[]
            #####################################################
            for sdate in stockbyweek.keys():
                if len(stockbyweek[sdate])>0:
                    print sdate
                    sdb=StockByDateDB(sdate,reg)
                    sdb.updatestockdata('stocklistweek',stockbyweek[sdate])
                    stockbyweek[sdate]=[] 
    #########################################################################
       
if __name__ == '__main__':
    starttime=datetime.datetime.now()
    print 'this is packStockTakeData  tasklist',starttime    
    sj=stock2011()    
#    sj.getdatabydate()
    sj.downloadstart()
    print 'download OK',starttime,(datetime.datetime.now()-starttime).seconds
    sj.dostockdb()
    print 'dostockdb OK',starttime,(datetime.datetime.now()-starttime).seconds
    sj.checkstockfile()
    print 'checkstockfile OK',starttime,(datetime.datetime.now()-starttime).seconds
    sj.trans2bydate()
    print 'start and end time:',starttime,(datetime.datetime.now()-starttime).seconds
#    clearallfile()